package com.spark.core.transformation

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

import scala.collection.mutable.ArrayBuffer

/**
 * mapPartitionsWithIndex
 * 可以拿到每个RDD中的分区，以及分区中的数据
 */
object Demo18_mapPartitionWithIndex {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
    conf.setMaster("local")
    conf.setAppName("mapPartitionWithIndex")
    val sc = new SparkContext(conf)
    sc.setLogLevel("error")

    val lines = sc.textFile("data/words", 5)

    val result: RDD[String] = lines.mapPartitionsWithIndex((index: Int, iter: Iterator[String]) => {
      val arr: ArrayBuffer[String] = ArrayBuffer[String]()
      iter.foreach(one => {
        arr.append(s"partition = 【$index】,value = $one")
      })
      arr.iterator
    })
    result.foreach(println)
  }
}
